{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37064bitf4f58e68e364450faf0aac3ea56fc105",
   "display_name": "Python 3.7.0 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 模块引入与初始化\n",
    "\n",
    "# import requests\n",
    "import time\n",
    "import threading\n",
    "# from bs4 import BeautifulSoup\n",
    "# from queue import Queue\n",
    "# import re\n",
    "# import os\n",
    "# from urllib.request import urlretrieve\n",
    "\n",
    "# 初始化\n",
    "gLock=threading.Lock()\n",
    "times=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 爬虫参数设置\n",
    "maxThtreadNum=5000000\n",
    "checkInterval=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 框架的函数\n",
    "\n",
    "#用于检测爬虫状态\n",
    "def chect(keys):\n",
    "    global times\n",
    "    global checkInterval\n",
    "\n",
    "    allKeys=len(keys)\n",
    "    preTime=time.time()\n",
    "    preTimes=times\n",
    "    speed=0\n",
    "    surplusTime='-'\n",
    "    while True:\n",
    "        if time.time()-preTime==0:\n",
    "            speed=0\n",
    "        else:\n",
    "            speed=(times-preTimes)/(time.time()-preTime)\n",
    "        if speed==0:\n",
    "            surplusTime='-'\n",
    "        else:\n",
    "            surplusTime=str(int((allKeys-times)/speed))\n",
    "        print('当前线程数%d,%d%%已完成,爬取速度%d次/秒，预计还需%s秒'%(len(threading.enumerate()),int((times/allKeys)*100),speed,surplusTime))\n",
    "        preTime=time.time()\n",
    "        preTimes=times\n",
    "        time.sleep(checkInterval)\n",
    "        if times==len(keys):\n",
    "            return\n",
    "\n",
    "\n",
    "#调配爬虫线程的函数\n",
    "def spiderThread(keys):\n",
    "    global gLock\n",
    "    global maxThtreadNum\n",
    "\n",
    "    #打开检测爬虫进度线程\n",
    "    t=threading.Thread(target=chect,args=(keys,))\n",
    "    t.start()\n",
    "\n",
    "    #打开所有爬虫线程\n",
    "    for key in keys:\n",
    "        #这是单线程爬虫的写法\n",
    "        # spider(key)\n",
    "\n",
    "        #这是多线程爬虫的写法\n",
    "        while True:\n",
    "            threadNum=len(threading.enumerate())#得到目前有多少线程\n",
    "            if threadNum<=maxThtreadNum:\n",
    "                t=threading.Thread(target=spider,args=(key,))\n",
    "                t.start()\n",
    "                break\n",
    "\n",
    "    #判断是否完成爬取            \n",
    "    while True:\n",
    "        if times==len(keys):\n",
    "            return\n",
    "        # print(str(int((times/len(keys))*100))+'%已完成')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#得到key列表\n",
    "def getKeys():\n",
    "    keys=[1]*100000\n",
    "    return keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#爬虫线程\n",
    "def spider(key):\n",
    "    #使用前必须声明全局变量\n",
    "    global gLock\n",
    "    global times\n",
    "\n",
    "    #以下用来数据提取\n",
    "    time.sleep(1)\n",
    "    \n",
    "    #以下用来数据存储\n",
    "    gLock.acquire()\n",
    "    \n",
    "    #times用来计数\n",
    "    times+=1\n",
    "    gLock.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "当前线程数7,0%已完成,爬取速度0次/秒，预计还需-秒\n当前线程数3327,0%已完成,爬取速度-99361次/秒，预计还需-1秒\n当前线程数4543,0%已完成,爬取速度0次/秒，预计还需100105秒\n当前线程数3350,3%已完成,爬取速度3315次/秒，预计还需29秒\n当前线程数3219,4%已完成,爬取速度4532次/秒，预计还需21秒\n当前线程数3490,6%已完成,爬取速度3339次/秒，预计还需27秒\n当前线程数3473,7%已完成,爬取速度3210次/秒，预计还需28秒\n当前线程数3633,10%已完成,爬取速度3477次/秒，预计还需25秒\n当前线程数3628,11%已完成,爬取速度3460次/秒，预计还需25秒\n当前线程数3626,13%已完成,爬取速度3618次/秒，预计还需23秒\n当前线程数3702,14%已完成,爬取速度3613次/秒，预计还需23秒\n当前线程数3652,17%已完成,爬取速度3614次/秒，预计还需22秒\n当前线程数3685,18%已完成,爬取速度3688次/秒，预计还需22秒\n当前线程数3708,21%已完成,爬取速度3643次/秒，预计还需21秒\n当前线程数3520,22%已完成,爬取速度3673次/秒，预计还需21秒\n当前线程数3491,24%已完成,爬取速度3698次/秒，预计还需20秒\n当前线程数3589,25%已完成,爬取速度3509次/秒，预计还需21秒\n当前线程数3725,28%已完成,爬取速度3479次/秒，预计还需20秒\n当前线程数3858,29%已完成,爬取速度3576次/秒，预计还需19秒\n当前线程数3622,31%已完成,爬取速度3713次/秒，预计还需18秒\n当前线程数3370,33%已完成,爬取速度3848次/秒，预计还需17秒\n当前线程数3579,35%已完成,爬取速度3609次/秒，预计还需17秒\n当前线程数3748,36%已完成,爬取速度3355次/秒，预计还需18秒\n当前线程数3562,39%已完成,爬取速度3569次/秒，预计还需17秒\n当前线程数3518,40%已完成,爬取速度3732次/秒，预计还需15秒\n当前线程数3598,42%已完成,爬取速度3552次/秒，预计还需16秒\n当前线程数3677,43%已完成,爬取速度3506次/秒，预计还需16秒\n当前线程数3699,46%已完成,爬取速度3585次/秒，预计还需14秒\n当前线程数3588,47%已完成,爬取速度3666次/秒，预计还需14秒\n当前线程数3473,49%已完成,爬取速度3682次/秒，预计还需13秒\n当前线程数3436,51%已完成,爬取速度3571次/秒，预计还需13秒\n当前线程数3431,53%已完成,爬取速度3449次/秒，预计还需13秒\n当前线程数3520,54%已完成,爬取速度3428次/秒，预计还需13秒\n当前线程数3548,56%已完成,爬取速度3406次/秒，预计还需12秒\n当前线程数3514,57%已完成,爬取速度3508次/秒，预计还需11秒\n当前线程数3580,60%已完成,爬取速度3536次/秒，预计还需11秒\n当前线程数3572,61%已完成,爬取速度3503次/秒，预计还需10秒\n当前线程数3516,64%已完成,爬取速度3567次/秒，预计还需10秒\n当前线程数3580,65%已完成,爬取速度3561次/秒，预计还需9秒\n当前线程数3571,67%已完成,爬取速度3504次/秒，预计还需9秒\n当前线程数3574,68%已完成,爬取速度3570次/秒，预计还需8秒\n当前线程数3657,71%已完成,爬取速度3555次/秒，预计还需8秒\n当前线程数3709,72%已完成,爬取速度3564次/秒，预计还需7秒\n当前线程数3728,74%已完成,爬取速度3646次/秒，预计还需6秒\n当前线程数3695,75%已完成,爬取速度3695次/秒，预计还需6秒\n当前线程数3582,78%已完成,爬取速度3718次/秒，预计还需5秒\n当前线程数3651,79%已完成,爬取速度3682次/秒，预计还需5秒\n当前线程数3678,82%已完成,爬取速度3572次/秒，预计还需5秒\n当前线程数3464,83%已完成,爬取速度3642次/秒，预计还需4秒\n当前线程数3393,85%已完成,爬取速度3665次/秒，预计还需3秒\n当前线程数3504,86%已完成,爬取速度3454次/秒，预计还需3秒\n当前线程数3412,89%已完成,爬取速度3384次/秒，预计还需3秒\n当前线程数3357,90%已完成,爬取速度3492次/秒，预计还需2秒\n当前线程数3436,92%已完成,爬取速度3397次/秒，预计还需2秒\n当前线程数3599,93%已完成,爬取速度3345次/秒，预计还需1秒\n当前线程数3562,95%已完成,爬取速度3423次/秒，预计还需1秒\n当前线程数2827,97%已完成,爬取速度3577次/秒，预计还需0秒\n当前线程数502,99%已完成,爬取速度3542次/秒，预计还需0秒\n共用了28.904181秒\n"
    }
   ],
   "source": [
    "# main\n",
    "startTime=time.time()\n",
    "spiderThread(getKeys())\n",
    "\n",
    "#这里对爬取到的数据进行处理\n",
    "\n",
    "\n",
    "print('共用了%f秒'%(time.time()-startTime))"
   ]
  }
 ]
}